import json
import os
import re
import sys
import threading
import time
import traceback
from ctypes import windll, c_char_p
from urllib import parse

import facade
import requests
from xjlibrary.mdatetime.mtime2 import MDateTimeUtils
from xjlibrary.our_file_dir import BaseDir
from xjlibrary.tools.BaseUrl import BaseUrl

curPath = BaseDir.get_file_dir_absolute(__file__)
TopPath = BaseDir.get_upper_dir(curPath, -3)
dbpath = BaseDir.get_upper_dir(curPath, -1)


class DiscernCode(object):
    """
    将识别验证码封装成一个类
    """

    def __init__(self, logger):
        self.appId = 6249  # 软件ＩＤ，开发者分成必要参数。登录开发者后台【我的软件】获得！
        self.appKey = b'439aa4305465df04545dad067bb31751'  # 软件密钥，开发者分成必要参数。登录开发者后台【我的软件】获得！
        self.YDMApi = windll.LoadLibrary('yundamaAPI-x64')
        # 注意这里是普通会员账号，不是开发者账号，注册地址 http://www.yundama.com/index/reg/user
        # 开发者可以联系客服领取免费调试题分
        # self.username = b'xujiang'
        # self.password = b'xujiang1994323'
        self.username = b'office'
        self.password = b'officeHelper$123'
        # 例：1004表示4位字母数字，不同类型收费不同。请准确填写，否则影响识别率。在此查询所有类型 http://www.yundama.com/price.html
        self.codetype = 3006
        # 分配30个字节存放识别结果
        self.result = c_char_p(b"                              ")
        # 验证码文件路径
        # self.filename = jpgpath.encode('utf-8')
        self.logger = logger
        # 识别超时时间 单位：秒
        self.timeout = 60

    def easy_decode_by_path(self, jpgpath):
        self.logger.info("正在开始一键识别{}".format(jpgpath))
        # 一键识别函数，无需调用 YDM_SetAppInfo 和 YDM_Login，适合脚本调用
        captchaId = self.YDMApi.YDM_EasyDecodeByPath(self.username,
                                                     self.password,
                                                     self.appId,
                                                     self.appKey,
                                                     jpgpath,
                                                     self.codetype,
                                                     self.timeout,
                                                     self.result)

        self.logger.info("一键识别：验证码ID：%d，识别结果：%s" % (captchaId, self.result.value))
        if int(captchaId) > 0:
            return True, str(self.result.value)
        else:
            self.logger.error("识别失败 请检查原因")
            return False, ""


class Downcited(object):
    # 可用ip
    listip = [
        "192.168.30.176:8207",
        "192.168.30.176:8012",
        "192.168.30.176:8021",
        "192.168.30.36:8041",  ##
        "192.168.30.176:8076",
        "192.168.30.36:8082",  ##
        "192.168.30.176:8165",
        "192.168.30.176:8171",
        "192.168.30.36:8182",  ##
        "192.168.30.176:8195",
        "192.168.30.176:8243",
        "192.168.30.176:8031",

    ]

    def __init__(self):
        self.url = "https://www.engineeringvillage.com/toolsinscopus/citedbycount.url"
        self.header = {
            "Accept": "application/json, text/javascript, */*; q=0.01",
            "Accept-Encoding": "gzip, deflate",
            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
            "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
            "Host": "www.engineeringvillage.com",
            "Origin": "https://www.engineeringvillage.com",
            "Referer": "https://www.engineeringvillage.com/search/expert.url",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36",
            "X-NewRelic-ID": "VQQAUldRCRAFUFFQBwgCUQ==",
            "X-Requested-With": "XMLHttpRequest"
        }
        self.configfile = BaseDir.get_new_path(dbpath, "db.ini")
        self.logger = facade.get_streamlogger()
        self.mysqlutils = facade.MysqlUtiles(self.configfile, "db", logger=self.logger)
        self.sn = requests.Session()
        self.t = int(round(time.time() * 1000))
        self.discern_code_time = int(time.time()) - 300
        self.Proxies = None
        self.init()

    def init(self):
        self.Form = {
            "usageOrigin": "searchform",
            "usageZone": "quicksearch",
            "editSearch": "",
            "isFullJsonResult": "true",
            "angularReq": "true",
            "CID": "searchSubmit",
            "searchtype": "Quick",
            "origin": "searchform",
            "category": "quicksearch",
            "section1": "NO-LIMIT",
            "searchWord1": "china",
            "database": "1",
            "yearselect": "yearrange",
            "startYear": "1884",
            "endYear": "2020",
            "updatesNo": "1",  # "4",
            "sort": "relevance",  # "sort": "relevance",  yr代表最新 relevance相关度排序
            "language": "NO-LIMIT",
            "doctype": "NO-LIMIT",
            "treatmentType": "NO-LIMIT",
            "searchStartTimestamp": str(int(round(time.time() * 1000))),
            "_": str(self.t)
        }

        self.headers = {
            "Accept": "application/json, text/javascript, */*; q=0.01",
            "Accept-Encoding": "gzip, deflate",
            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
            "Connection": "keep-alive",
            "Content-Type": "application/json",
            "Host": "www.engineeringvillage.com",
            "Origin": "https://www.engineeringvillage.com",
            "Referer": "https://www.engineeringvillage.com/search/quick.url?usageZone=evlogo&usageOrigin=header",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36",
            "X-NewRelic-ID": "VQQAUldRCRAFUFFQBwgCUQ==",
            "X-Requested-With": "XMLHttpRequest"
        }

    def set_proxy(self, proxy):
        self.Proxies = {
            "http": proxy,
            "https": proxy  # key是指目标网站的协议
        }

    def discern_code(self, jpgpath):
        """
        识别验证码通过云打码
        :return:
        """
        if int(time.time()) - self.discern_code_time > 60 * 5:  #
            boolresult, code = DiscernCode(self.logger).easy_decode_by_path(jpgpath)
            BaseDir.single_add_file("./code.txt",
                                    "***{}***{}***code is :{}\n".format(MDateTimeUtils.get_beijin_date_strins(),
                                                                        self.Proxies["http"], code))
            self.discern_code_time = int(time.time())
            if boolresult:
                code = code.replace("b'", "").replace("'", "")
                return True, code
            else:
                return False, ""
        else:
            self.logger.info("时间还没到休息60秒 差时为{} 应该相隔5分钟".format(int(time.time()) - self.discern_code_time))
            time.sleep(60)
            return False, ""

    def get_searchid(self, ac):
        self.logger.info('DownOneref')

        # 如果目录存在表示下载过 但也有可能是下载一部分后中断了
        # outDir = os.path.join(dirPath, ac)
        # if os.path.exists(outDir):
        #     return

        def init_home():
            url = r'https://www.engineeringvillage.com/home.url'
            header = {
                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3",
                "Accept-Encoding": "gzip, deflate",
                "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
                "Host": "www.engineeringvillage.com",
                "Upgrade-Insecure-Requests": "1",
                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36"
            }
            BoolResult, errString, r = facade.BaseRequest(url,
                                                          sn=self.sn,
                                                          headers=header,
                                                          verify=False,
                                                          proxies=self.Proxies,
                                                          timeout=60)
            self.logger.info(errString)
            if not BoolResult:
                return False
            self.logger.info("首页访问完成")
            print(r.url)
            BaseDir.single_write_file(r.text, "./test.html")
            if r.text.find("Chongqing University Library, General Access") > -1:
                obj = re.search(r'csrfToken\.value = "(.*?)"', r.text)
                csrfToken = obj.group(1)
                print(csrfToken)
                url = "https://www.engineeringvillage.com/customer/authenticate.url"
                postdate = {
                    "path_choice": "13022631",
                    "remember_path_flag": "true",
                    "origin": "pathChoice",
                    "zone": "main",
                    "auth_type": "CANDIDATE_PATH",
                    "csrfToken": csrfToken
                }
                BoolResult, errString, r = facade.BaseRequestPost(url=url,
                                                                  sn=self.sn,
                                                                  data=postdate,
                                                                  proxies=self.Proxies,
                                                                  allow_redirects=True,
                                                                  endstring="",
                                                                  verify=False,
                                                                  timeout=60)
                if BoolResult:
                    print("suceess")
                else:
                    print("failed")
                    sys.exit(-1)

        form = self.Form
        # 5846789  17977686
        form['searchWord1'] = "{}".format(ac)
        # form['startYear'] = str(year)
        #         # form['endYear'] = str(year)
        url = r'https://www.engineeringvillage.com/search/submit.url'
        url = url + "?" + BaseUrl.dicts_to_url(form)
        BoolResult, errString, r = facade.BaseRequest(url=url,
                                                      sn=self.sn,
                                                      headers=self.headers,
                                                      proxies=self.Proxies,
                                                      allow_redirects=True,
                                                      endstring="",
                                                      verify=False,
                                                      timeout=60)
        self.logger.info(errString)

        # self.logger.info(r.status_code)
        if errString == "request":
            return False
        # if r and r.status_code == 200:
        #
        if not BoolResult:
            if r.status_code == 400:
                imagetextenc = json.loads(r.text)["imagetextenc"]
                url = "https://www.engineeringvillage.com/rest/captcha/image?imagetextenc={}".format(imagetextenc)
                BoolResult, errString, r = facade.BaseRequest(url=url,
                                                              sn=self.sn,
                                                              proxies=self.Proxies,
                                                              allow_redirects=True,
                                                              endstring="",
                                                              timeout=60)
                if BoolResult:
                    jpgpath = BaseDir.get_new_path(curPath, str(os.getpid()) + ".jpg")
                    BaseDir.single_write_wb_file(r.content, jpgpath)
                else:
                    print("获取验证码失败")
                    return False
                # code = input("please input code:")
                jpgpath = BaseDir.get_new_path(curPath, str(os.getpid()) + ".jpg")
                jpgpath = jpgpath.encode('utf-8')
                bools, code = self.discern_code(jpgpath)
                if not bools:
                    return False
                url = "https://www.engineeringvillage.com/rest/captcha/verify?imagetextenc={}&userentry={}".format(
                    imagetextenc, code)
                BoolResult, errString, r = facade.BaseRequest(url=url,
                                                              sn=self.sn,
                                                              proxies=self.Proxies,
                                                              allow_redirects=True,
                                                              endstring="",
                                                              timeout=60)
                if BoolResult:
                    url = r'https://www.engineeringvillage.com/search/submit.url'
                    url = url + "?" + BaseUrl.dicts_to_url(form)
                    BoolResult, errString, r = facade.BaseRequest(url=url,
                                                                  sn=self.sn,
                                                                  proxies=self.Proxies,
                                                                  allow_redirects=True,
                                                                  endstring="",
                                                                  timeout=60)
                    if not BoolResult:
                        print("验证后失败")
                        return False
                else:
                    print("验证失败")
                    return False
            else:
                return False
        self.logger.info("搜索成功")
        print(r.url)
        BaseDir.single_write_file(r.text, "./test2.html")
        try:
            param_dict = parse.parse_qs(parse.urlparse(r.url).query)
            self.SEARCHID = param_dict['SEARCHID'][0]
            dicts = json.loads(r.text)
            self.sessionId = dicts["searchMetaData"]["searchesEntity"]["sessionId"]
            self.logger.info("SEARCHID is:" + self.SEARCHID)
            self.logger.info("sessionId is:" + self.sessionId)
            return True
        except:
            init_home()
            return False

    # def get_abs(self):
    #     """
    #     下载abs是为了获取sessionid
    #     :return:
    #     """

    def select(self, row):
        print(row[1])
        url = "https://www.engineeringvillage.com/search/doc/abstract.url"
        dicturl = {
            "content": "true",
            "pageType": "quickSearch",
            "usageZone": "resultslist",
            "usageOrigin": "searchresults",
            "searchtype": "Quick",
            "SEARCHID": self.SEARCHID,
            "DOCINDEX": "1",
            "ignore_docid": row[1],
            "database": "1",
            "format": "quickSearchAbstractFormat",
            "tagscope": "",
            "displayPagination": "yes"
        }

        url = url + "?" + BaseUrl.dicts_to_url(dicturl)

        headers = {
            "Accept": "application/json, text/javascript, */*; q=0.01",
            "Accept-Encoding": "gzip, deflate",
            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
            "Content-Type": "application/json",
            "Host": "www.engineeringvillage.com",
            "Upgrade-Insecure-Requests": "1",
            "X-NewRelic-ID": "VQQAUldRCRAFUFFQBwgCUQ==",
            "X-Requested-With": "XMLHttpRequest",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36"
        }

        BoolResult, errString, r = facade.BaseRequest(url,
                                                      sn=self.sn,
                                                      proxies=self.Proxies,
                                                      allow_redirects=True,
                                                      endstring="",
                                                      mark="citedby",
                                                      timeout=60,
                                                      headers=headers
                                                      )
        if BoolResult:
            jsondicts = json.loads(r.text)
            dicts = jsondicts["result"]["citedby"]
            sessionid = jsondicts["sessionid"]
            # 基金的两个信息
            fundetails = jsondicts["result"]["abstractrecord"]["fundingDetails"]
            gtt = jsondicts["result"]["gtt"]
            if gtt or fundetails:
                fundicts = {}
                fundicts["accnum"] = row[0]
                fundicts["details"] = fundetails
                fundicts["gtt"] = gtt
                funstrings = json.dumps(fundicts, ensure_ascii=False)
                BaseDir.single_add_file("./fun_"+str(os.getpid())+"_"+str(threading.get_ident())+".big_json", funstrings + "\n")

        else:
            print("下载失败")
            sys.exit(-1)

        print(dicts)

        listcite2 = []
        # dicts = json.loads(row[2])
        # dic = collections.OrderedDict()
        # # dicts = json.loads(row[2])
        # #  dicts["md5"] 这个参数不是固定的
        # dic["issn"] = dicts["issn"]
        # dic["isbn"] = dicts["isbn"]
        # dic["isbn13"] = dicts["isbn13"]
        # dic["doi"] = dicts["doi"]
        # dic["pii"] = dicts["pii"]
        # dic["vol"] = dicts["firstvolume"]
        # dic["issue"] = dicts["firstissue"]
        # dic["page"] = dicts["firstpage"]
        # dic["an"] = dicts["an"]
        # dic["security"] = dicts["md5"]
        # dic["sid"] = sessionid # sessionid

        # dicts = json.loads(row[1])
        dicts["security"] = dicts["md5"]
        dicts["vol"] = dicts["firstvolume"]
        dicts["issue"] = dicts["firstissue"]
        dicts["page"] = dicts["firstpage"]
        dicts["sid"] = sessionid

        del dicts["md5"]
        del dicts["firstissue"]
        del dicts["firstvolume"]
        del dicts["firstpage"]
        del dicts["doiencoded"]

        # strings = "[{"
        # for k, v in dic.items():
        #     strings = strings + "\"" + k + "\":\"" + v + "\","
        # strings = strings.strip(",")
        # strings = strings + "}]"

        listcite2.append(dicts)
        strings = str(listcite2).replace("'", "\"")
        postdate = {
            "citedby": strings
        }
        self.logger.info("AccessionNumber is:" + row[0])
        self.logger.info("postdate is:" + str(postdate))
        # self.headers["Referer"] = r.url
        BoolResult, errString, r = facade.BaseRequestPost(self.url,
                                                          sn=self.sn,
                                                          proxies=self.Proxies,
                                                          allow_redirects=True,
                                                          endstring="",
                                                          mark="results",
                                                          timeout=60,
                                                          data=postdate,
                                                          # headers=self.headers
                                                          )
        if BoolResult:
            BaseDir.single_write_file(r.text, "./test3.html")
            print(r.text)
            result = json.loads(r.text)["results"]
            print(result)
            print(type(result))
            if result:
                print("******************")
                eid = result[0]["EID"]
                count = result[0]["COUNT"]
                accnumber = result[0]["ID"]
                sid = result[0]["SID"]
                citedtime = MDateTimeUtils.get_today_date_strings()
                sql = "update articlenew set `cited_cnt`='{}',`eid`='{}',`sid`='{}',citedtime='{}' where AccessionNumber='{}'".format(
                    count, eid, sid, citedtime, accnumber)
                print(sql)
                self.mysqlutils.ExeSqlToDB(sql)
            else:
                sql = "update articlenew set eid='-1' where AccessionNumber='{}'".format(row[0])
                self.mysqlutils.ExeSqlToDB(sql)


        else:
            print("下载错误")

"""
update `articlenew` set eid = NULL WHERE eid=''
"""
if __name__ == "__main__":
    down = Downcited()
    down.set_proxy("192.168.30.176:8012")
    count = 0
    while True:
        sql = "SELECT AccessionNumber,docid,citemsg FROM `articlenew` WHERE eid is null limit 200"  #
        rows = down.mysqlutils.SelectFromDB(sql)
        Listac = []
        for row in rows:
            Listac.append(row[0])
        sql = "update articlenew set eid='' where AccessionNumber in {}".format(tuple(Listac))
        down.mysqlutils.ExeSqlToDB(sql)
        for row in rows:
            count = count + 1
            down.logger.info("count is:"+str(count))
            try:
                if down.get_searchid(row[0]):
                    down.select(row)
                    if count > 8:
                        count = 0
                        down.sn = requests.session()
                    # time.sleep(5)
            except:
                traceback.print_exc()
                count = 0
